From: Jonathan Dieter <jdieter@gmail.com>
Date: Tue, 10 Apr 2018 12:57:19 +0000 (+0300)
Subject: Massive work changing API to support zck_read and zck_write functions
X-Git-Tag: archive/raspbian/1.1.9+ds1-1+rpi1~1^2~333
X-Git-Url: https://dgit.raspbian.org/%22http://www.example.com/cgi/%22/%22http:/www.example.com/cgi/%22?a=commitdiff_plain;h=14743ec98517460dda611017b4f02305231a5353;p=zchunk.git

Massive work changing API to support zck_read and zck_write functions

Signed-off-by: Jonathan Dieter <jdieter@gmail.com>
---

diff --git a/src/lib/comp/comp.c b/src/lib/comp/comp.c
index c1716e2..e278e99 100644
--- a/src/lib/comp/comp.c
+++ b/src/lib/comp/comp.c
@@ -38,14 +38,50 @@
 
 #define BLK_SIZE 32768
 #define VALIDATE(f)     if(!f) { \
-                            zck_log(ZCK_LOG_ERROR, "zckCtx not initialized\n"); \
+                            zck_log(ZCK_LOG_ERROR, \
+                                    "zckCtx not initialized\n"); \
                             return False; \
                         }
 
+#define VALIDATE_READ(f)    VALIDATE(f); \
+                            if(f->mode != ZCK_MODE_READ) { \
+                                zck_log(ZCK_LOG_ERROR, \
+                                        "zckCtx not opened for reading\n"); \
+                                return False; \
+                            }
+
+#define VALIDATE_WRITE(f)   VALIDATE(f); \
+                            if(f->mode != ZCK_MODE_WRITE) { \
+                                zck_log(ZCK_LOG_ERROR, \
+                                        "zckCtx not opened for writing\n"); \
+                                return False; \
+                            }
+
+#define VALIDATE_SIZE(f)    if(!f) { \
+                                zck_log(ZCK_LOG_ERROR, \
+                                        "zckComp not initialized\n"); \
+                                return -1; \
+                            }
+
+#define VALIDATE_READ_SIZE(f)   VALIDATE_SIZE(f); \
+                                if(f->mode != ZCK_MODE_READ) { \
+                                    zck_log(ZCK_LOG_ERROR, \
+                                            "zckComp not opened for reading\n"); \
+                                    return -1; \
+                                }
+
+#define VALIDATE_WRITE_SIZE(f)  VALIDATE_SIZE(f); \
+                                if(f->mode != ZCK_MODE_WRITE) { \
+                                    zck_log(ZCK_LOG_ERROR, \
+                                            "zckComp not opened for writing\n"); \
+                                    return -1; \
+                                }
+
 static char unknown[] = "Unknown(\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";
 
 const static char *COMP_NAME[] = {
     "no",
+    "Unknown (1)",
     "zstd"
 };
 
@@ -53,8 +89,6 @@ int zck_comp_init(zckCtx *zck) {
     VALIDATE(zck);
 
     zckComp *comp = &(zck->comp);
-    char *dst = NULL;
-    size_t dst_size = 0;
 
     if(zck->comp.started) {
         zck_log(ZCK_LOG_ERROR, "Compression already initialized\n");
@@ -65,13 +99,17 @@ int zck_comp_init(zckCtx *zck) {
         zck_log(ZCK_LOG_ERROR, "Invalid dictionary configuration\n");
         return False;
     }
+    zck_log(ZCK_LOG_DEBUG, "Initializing %s compression\n",
+            zck_comp_name_from_type(comp->type));
     if(!zck->comp.init(&(zck->comp)))
         return False;
-
     if(zck->temp_fd) {
         if(zck->comp.dict) {
-            if(!zck->comp.compress(comp, zck->comp.dict, zck->comp.dict_size, &dst,
-                                   &dst_size, 0))
+            char *dst = NULL;
+            size_t dst_size = 0;
+
+            if(zck->comp.compress(comp, zck->comp.dict, zck->comp.dict_size,
+                                  &dst, &dst_size, 0) < 0)
                 return False;
             if(!write_data(zck->temp_fd, dst, dst_size)) {
                 free(dst);
@@ -82,7 +120,7 @@ int zck_comp_init(zckCtx *zck) {
             dst = NULL;
             dst_size = 0;
 
-            if(!zck->comp.end_chunk(comp, &dst, &dst_size, 0))
+            if(!zck->comp.end_cchunk(comp, &dst, &dst_size, 0))
                 return False;
             if(!write_data(zck->temp_fd, dst, dst_size)) {
                 free(dst);
@@ -101,15 +139,34 @@ int zck_comp_init(zckCtx *zck) {
     return True;
 }
 
-int zck_comp_close(zckCtx *zck) {
+int zck_comp_reset(zckCtx *zck) {
     VALIDATE(zck);
 
     zck->comp.started = 0;
+    if(zck->comp.dc_data) {
+        free(zck->comp.dc_data);
+        zck->comp.dc_data = NULL;
+        zck->comp.dc_data_loc = 0;
+        zck->comp.dc_data_size = 0;
+    }
     if(zck->comp.close == NULL)
         return True;
     return zck->comp.close(&(zck->comp));
 }
 
+int zck_comp_close(zckCtx *zck) {
+    VALIDATE(zck);
+    zck_log(ZCK_LOG_DEBUG, "Closing compression\n");
+    if(zck->comp.data) {
+        free(zck->comp.data);
+        zck->comp.data = NULL;
+        zck->comp.data_size = 0;
+        zck->comp.data_loc = 0;
+        zck->comp.data_idx = NULL;
+    }
+    return zck_comp_reset(zck);
+}
+
 int zck_set_compression_type(zckCtx *zck, int type) {
     VALIDATE(zck);
 
@@ -165,85 +222,239 @@ int zck_set_comp_parameter(zckCtx *zck, int option, void *value) {
 }
 
 const char *zck_comp_name_from_type(int comp_type) {
-    if(comp_type > 1) {
+    if(comp_type > 2) {
         snprintf(unknown+8, 21, "%i)", comp_type);
         return unknown;
     }
     return COMP_NAME[comp_type];
 }
 
-int zck_write(zckCtx *zck, const char *src, const size_t src_size) {
-    VALIDATE(zck);
+size_t zck_comp_read_from_dc(zckComp *comp, char *dst, size_t dst_size) {
+    VALIDATE_SIZE(comp);
+    VALIDATE_SIZE(dst);
+
+    size_t dl_size = dst_size;
+    if(dl_size > comp->dc_data_size - comp->dc_data_loc)
+        dl_size = comp->dc_data_size - comp->dc_data_loc;
+    memcpy(dst, comp->dc_data+comp->dc_data_loc, dl_size);
+    comp->dc_data_loc += dl_size;
+    if(dl_size > 0)
+        zck_log(ZCK_LOG_DEBUG, "Reading %lu bytes from decompressed buffer\n",
+                dl_size);
+    return dl_size;
+}
 
-    if(!zck->comp.started && !zck_comp_init(zck))
+int zck_comp_add_to_dc(zckComp *comp, const char *src, size_t src_size) {
+    VALIDATE(comp);
+    VALIDATE(src);
+
+    /* Get rid of any already read data and allocate space for new data */
+    char *temp = zmalloc(comp->dc_data_size - comp->dc_data_loc + src_size);
+    if(temp == NULL) {
+        zck_log(ZCK_LOG_ERROR, "Unable to reallocate %lu bytes\n",
+                comp->dc_data_size - comp->dc_data_loc + src_size);
         return False;
+    }
+    if(comp->dc_data_loc != 0)
+        zck_log(ZCK_LOG_DEBUG, "Freeing %lu bytes from decompressed buffer\n",
+                comp->dc_data_loc);
+    zck_log(ZCK_LOG_DEBUG, "Adding %lu bytes to decompressed buffer\n",
+            src_size);
+    memcpy(temp, comp->dc_data + comp->dc_data_loc,
+           comp->dc_data_size - comp->dc_data_loc);
+    free(comp->dc_data);
+    comp->dc_data_size -= comp->dc_data_loc;
+    comp->dc_data_loc = 0;
+    comp->dc_data = temp;
+
+    /* Copy new uncompressed data into comp */
+    memcpy(comp->dc_data + comp->dc_data_size, src, src_size);
+    comp->dc_data_size += src_size;
+    return True;
+}
+
+int zck_comp_add_to_data(zckComp *comp, const char *src, size_t src_size) {
+    VALIDATE(comp);
+    VALIDATE(src);
+    comp->data = realloc(comp->data, comp->data_size + src_size);
+    if(comp->data == NULL) {
+        zck_log(ZCK_LOG_ERROR, "Unable to reallocate %lu bytes\n",
+                comp->data_size + src_size);
+        return False;
+    }
+    zck_log(ZCK_LOG_DEBUG, "Adding %lu bytes to compressed buffer\n",
+        src_size);
+    memcpy(comp->data + comp->data_size, src, src_size);
+    comp->data_size += src_size;
+    comp->data_loc += src_size;
+    return True;
+}
+
+ssize_t zck_write(zckCtx *zck, const char *src, const size_t src_size) {
+    VALIDATE_WRITE_SIZE(zck);
+
+    if(!zck->comp.started && !zck_comp_init(zck))
+        return -1;
 
     if(src_size == 0)
-        return True;
+        return 0;
 
     char *dst = NULL;
     size_t dst_size = 0;
-    if(!zck->comp.compress(&(zck->comp), src, src_size, &dst, &dst_size, 1))
-        return False;
+    if(zck->comp.compress(&(zck->comp), src, src_size, &dst, &dst_size, 1) < 0)
+        return -1;
     if(dst_size > 0 && !write_data(zck->temp_fd, dst, dst_size)) {
         free(dst);
-        return False;
+        return -1;
     }
     if(!zck_index_add_to_chunk(zck, dst, dst_size, src_size)) {
         free(dst);
-        return False;
+        return -1;
     }
     free(dst);
-    return True;
+    return src_size;
 }
 
-int zck_end_chunk(zckCtx *zck) {
-    VALIDATE(zck);
+ssize_t zck_end_chunk(zckCtx *zck) {
+    VALIDATE_WRITE_SIZE(zck);
 
     if(!zck->comp.started && !zck_comp_init(zck))
-        return False;
+        return -1;
 
     /* No point in compressing empty data */
-    if(zck->comp.data_size == 0)
-        return True;
+    if(zck->comp.dc_data_size == 0)
+        return 0;
 
+    size_t data_size = zck->comp.dc_data_size;
     char *dst = NULL;
     size_t dst_size = 0;
-    if(!zck->comp.end_chunk(&(zck->comp), &dst, &dst_size, 1))
-        return False;
+    if(!zck->comp.end_cchunk(&(zck->comp), &dst, &dst_size, 1))
+        return -1;
     if(dst_size > 0 && !write_data(zck->temp_fd, dst, dst_size)) {
         free(dst);
-        return False;
+        return -1;
     }
     if(!zck_index_add_to_chunk(zck, dst, dst_size, 0)) {
         free(dst);
-        return False;
+        return -1;
     }
     if(!zck_index_finish_chunk(zck)) {
         free(dst);
-        return False;
+        return -1;
     }
     free(dst);
-    return True;
+    return data_size;
 }
 
-int zck_read(zckCtx *zck, const char *src, const size_t src_size, char **dst,
-             size_t dst_size) {
-    VALIDATE(zck);
+ssize_t comp_end_dchunk(zckCtx *zck, int use_dict, size_t fd_size) {
+    ssize_t rb = zck->comp.end_dchunk(&(zck->comp), use_dict, fd_size);
+    if(!zck_validate_current_chunk(zck))
+        return -1;
+    zck->comp.data_loc = 0;
+    zck->comp.data_idx = zck->comp.data_idx->next;
+    zck_hash_init(&(zck->check_chunk_hash), &(zck->chunk_hash_type));
+    return rb;
+}
 
-    zckComp *comp = &(zck->comp);
-    *dst = NULL;
+ssize_t comp_read(zckCtx *zck, char *dst, size_t dst_size, int use_dict) {
+    VALIDATE_READ_SIZE(zck);
 
     if(!zck->comp.started) {
         zck_log(ZCK_LOG_ERROR, "Compression hasn't been initialized yet\n");
-        return False;
+        return -1;
     }
 
-    if(src_size == 0)
-        return True;
+    if(dst_size == 0)
+        return 0;
 
-    if(!zck->comp.decompress(comp, src, src_size, dst, dst_size, 1))
+    size_t dc = 0;
+    char *src = zmalloc(dst_size - dc);
+    if(src == NULL) {
+        zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", dst_size-dc);
         return False;
+    }
+    int finished_rd = False;
+    int finished_dc = False;
+    zck_log(ZCK_LOG_DEBUG, "Trying to read %lu bytes\n", dst_size);
+    while(dc < dst_size) {
+        /* Get bytes from decompressed buffer */
+        ssize_t rb = zck_comp_read_from_dc(&(zck->comp), dst+dc, dst_size-dc);
+        if(rb < 0)
+            goto zck_read_error;
+        dc += rb;
+        if(dc == dst_size)
+            break;
+        if(rb > 0)
+            continue;
+        if(finished_dc || zck->comp.data_eof)
+            break;
+
+        /* Decompress compressed buffer into decompressed buffer */
+        size_t dc_data_size = zck->comp.dc_data_size;
+        size_t dc_data_loc = zck->comp.dc_data_loc;
+        if(!zck->comp.decompress(&(zck->comp), use_dict))
+            goto zck_read_error;
+
+        /* Check whether we decompressed more data */
+        if(zck->comp.dc_data_size != dc_data_size ||
+           zck->comp.dc_data_loc != dc_data_loc)
+            continue;
+
+        /* End decompression chunk if we're on a chunk boundary */
+        if(zck->comp.data_idx == NULL) {
+            zck->comp.data_idx = zck->index.first;
+            zck_hash_init(&(zck->check_chunk_hash), &(zck->chunk_hash_type));
+            zck->comp.data_loc = 0;
+        }
+        if(zck->comp.data_loc == zck->comp.data_idx->comp_length) {
+            if(comp_end_dchunk(zck, use_dict, zck->comp.data_idx->length) < 0)
+                return -1;
+            if(zck->comp.data_idx == NULL) {
+                if(!zck_validate_file(zck))
+                    goto zck_hash_error;
+                zck->comp.data_eof = True;
+            }
+            continue;
+        }
 
-    return True;
+        /* If we finished reading and we've reached here, we're done
+         * decompressing */
+        if(finished_rd) {
+            finished_dc = True;
+            continue;
+        }
+
+        /* Make sure we don't read beyond current chunk length */
+        size_t rs = dst_size;
+        if(zck->comp.data_loc + rs > zck->comp.data_idx->comp_length)
+            rs = zck->comp.data_idx->comp_length - zck->comp.data_loc;
+
+        /* Decompressed buffer is empty, so read data from file and fill
+         * compressed buffer */
+        rb = read_data(zck->fd, src, rs);
+        if(rb < 0)
+            goto zck_read_error;
+        if(rb < rs) {
+            zck_log(ZCK_LOG_DEBUG, "EOF\n");
+            finished_rd = True;
+        }
+        zck_hash_update(&(zck->check_full_hash), src, rb);
+        zck_hash_update(&(zck->check_chunk_hash), src, rb);
+        if(!zck_comp_add_to_data(&(zck->comp), src, rb))
+            goto zck_read_error;
+    }
+    free(src);
+    return dc;
+zck_read_error:
+    free(src);
+    return -1;
+zck_hash_error:
+    free(src);
+    return -2;
+}
+
+ssize_t zck_read(zckCtx *zck, char *dst, size_t dst_size) {
+    VALIDATE_READ_SIZE(zck);
+
+    return comp_read(zck, dst, dst_size, 1);
 }
diff --git a/src/lib/comp/nocomp/nocomp.c b/src/lib/comp/nocomp/nocomp.c
index 0b67206..ca844e7 100644
--- a/src/lib/comp/nocomp/nocomp.c
+++ b/src/lib/comp/nocomp/nocomp.c
@@ -31,69 +31,74 @@
 
 #include "zck_private.h"
 
-static int zck_nocomp_init(zckComp *comp) {
+static int init(zckComp *comp) {
     return True;
 }
 
-static int zck_nocomp_end_chunk(zckComp *comp, char **dst, size_t *dst_size,
-                                int use_dict) {
+static int end_cchunk(zckComp *comp, char **dst, size_t *dst_size,
+                     int use_dict) {
     *dst = NULL;
     *dst_size = 0;
 
     return True;
 }
 
-static int zck_nocomp_comp(zckComp *comp, const char *src, const size_t src_size,
-                           char **dst, size_t *dst_size, int use_dict) {
+static ssize_t compress(zckComp *comp, const char *src, const size_t src_size,
+                        char **dst, size_t *dst_size, int use_dict) {
     *dst = zmalloc(src_size);
     if(dst == NULL) {
         zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", src_size);
-        return False;
+        return -1;
     }
 
     memcpy(*dst, src, src_size);
     *dst_size = src_size;
 
-    return True;
+    return *dst_size;
 }
 
-static int zck_nocomp_decomp(zckComp *comp, const char *src,
-                             const size_t src_size, char **dst, size_t dst_size,
-                             int use_dict) {
-    *dst = zmalloc(src_size);
-    if(dst == NULL) {
-        zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", src_size);
+
+static int decompress(zckComp *comp, const int use_dict) {
+    char *src = comp->data;
+    char src_size = comp->data_size;
+    comp->data = NULL;
+    comp->data_size = 0;
+    if(!zck_comp_add_to_dc(comp, src, src_size)) {
+        free(src);
         return False;
     }
+    free(src);
+    return True;
+}
 
-    memcpy(*dst, src, src_size);
-
+static int end_dchunk(zckComp *comp, const int use_dict, const size_t fd_size) {
     return True;
 }
 
-static int zck_nocomp_close(zckComp *comp) {
+static int close(zckComp *comp) {
     return True;
 }
 
 /* Nocomp doesn't support any parameters, so return error if setting a parameter
  * was attempted */
-static int zck_nocomp_set_parameter(zckComp *comp, int option, void *value) {
+static int set_parameter(zckComp *comp, int option, void *value) {
     zck_log(ZCK_LOG_ERROR, "Invalid compression parameter for ZCK_COMP_NONE\n");
     return False;
 }
 
 /* No default parameters to set when there's no compression */
-static int zck_nocomp_set_default_parameters(zckComp *comp) {
+static int set_default_parameters(zckComp *comp) {
     return True;
 }
 
 int zck_nocomp_setup(zckComp *comp) {
-    comp->init = zck_nocomp_init;
-    comp->set_parameter = zck_nocomp_set_parameter;
-    comp->compress = zck_nocomp_comp;
-    comp->end_chunk = zck_nocomp_end_chunk;
-    comp->decompress = zck_nocomp_decomp;
-    comp->close = zck_nocomp_close;
+    comp->init = init;
+    comp->set_parameter = set_parameter;
+    comp->compress = compress;
+    comp->end_cchunk = end_cchunk;
+    comp->decompress = decompress;
+    comp->end_dchunk = end_dchunk;
+    comp->close = close;
     comp->type = ZCK_COMP_NONE;
-    return zck_nocomp_set_default_parameters(comp);
+    return set_default_parameters(comp);
 }
diff --git a/src/lib/comp/zstd/zstd.c b/src/lib/comp/zstd/zstd.c
index 51a9ed2..9c33f56 100644
--- a/src/lib/comp/zstd/zstd.c
+++ b/src/lib/comp/zstd/zstd.c
@@ -61,28 +61,25 @@ static int init(zckComp *comp) {
 /* The zstd compression format doesn't allow streaming compression with a dict
  * unless you statically link to it.  If we have a dict, we do pseudo-streaming
  * compression where we buffer the data until the chunk ends. */
-static int compress(zckComp *comp, const char *src, const size_t src_size,
-                    char **dst, size_t *dst_size, int use_dict) {
-    if(comp->data == NULL)
-        comp->data = zmalloc(src_size);
-    else
-        comp->data = realloc(comp->data, comp->data_size + src_size);
-    if(comp->data == NULL) {
+static ssize_t compress(zckComp *comp, const char *src, const size_t src_size,
+                        char **dst, size_t *dst_size, int use_dict) {
+    comp->dc_data = realloc(comp->dc_data, comp->dc_data_size + src_size);
+    if(comp->dc_data == NULL) {
         zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n",
-                comp->data_size + src_size);
-        return False;
+                comp->dc_data_size + src_size);
+        return -1;
     }
-    memcpy(comp->data + comp->data_size, src, src_size);
+    memcpy(comp->dc_data + comp->dc_data_size, src, src_size);
     *dst = NULL;
     *dst_size = 0;
-    comp->data_size += src_size;
-    return True;
+    comp->dc_data_size += src_size;
+    return 0;
 }
 
-static int end_chunk(zckComp *comp, char **dst, size_t *dst_size,
-                     int use_dict) {
+static int end_cchunk(zckComp *comp, char **dst, size_t *dst_size,
+                      int use_dict) {
     VALIDATE(comp);
-    size_t max_size = ZSTD_compressBound(comp->data_size);
+    size_t max_size = ZSTD_compressBound(comp->dc_data_size);
     if(ZSTD_isError(max_size)) {
         zck_log(ZCK_LOG_ERROR, "zstd compression error: %s\n",
                 ZSTD_getErrorName(max_size));
@@ -97,15 +94,16 @@ static int end_chunk(zckComp *comp, char **dst, size_t *dst_size,
 
     if(use_dict && comp->cdict_ctx) {
         *dst_size = ZSTD_compress_usingCDict(comp->cctx, *dst, max_size,
-                                             comp->data, comp->data_size,
+                                             comp->dc_data, comp->dc_data_size,
                                              comp->cdict_ctx);
     } else {
-        *dst_size = ZSTD_compressCCtx(comp->cctx, *dst, max_size, comp->data,
-                                      comp->data_size, comp->level);
+        *dst_size = ZSTD_compressCCtx(comp->cctx, *dst, max_size, comp->dc_data,
+                                      comp->dc_data_size, comp->level);
     }
-    free(comp->data);
-    comp->data = NULL;
-    comp->data_size = 0;
+    free(comp->dc_data);
+    comp->dc_data = NULL;
+    comp->dc_data_size = 0;
+    comp->dc_data_loc = 0;
     if(ZSTD_isError(*dst_size)) {
         zck_log(ZCK_LOG_ERROR, "zstd compression error: %s\n",
                 ZSTD_getErrorName(*dst_size));
@@ -114,31 +112,49 @@ static int end_chunk(zckComp *comp, char **dst, size_t *dst_size,
     return True;
 }
 
-static int decompress(zckComp *comp, const char *src, const size_t src_size,
-                      char **dst, size_t dst_size, int use_dict) {
+static int decompress(zckComp *comp, const int use_dict) {
     VALIDATE(comp);
 
-    *dst = zmalloc(dst_size);
-    if(dst == NULL) {
-        zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", dst_size);
-        return False;
-    }
+    return True;
+}
+
+static int end_dchunk(zckComp *comp, const int use_dict, const size_t fd_size) {
+    char *src = comp->data;
+    size_t src_size = comp->data_size;
+    comp->data = NULL;
+    comp->data_size = 0;
+
+    char *dst = zmalloc(fd_size);
+    if(dst == NULL)
+        goto decomp_error_1;
 
     size_t retval;
-    if(use_dict && comp->ddict_ctx)
-        retval = ZSTD_decompress_usingDDict(comp->dctx, *dst, dst_size, src,
+    zck_log(ZCK_LOG_DEBUG, "Decompressing %lu bytes to %lu bytes\n", src_size,
+            fd_size);
+    if(use_dict && comp->ddict_ctx) {
+        zck_log(ZCK_LOG_DEBUG, "Running decompression using dict\n");
+        retval = ZSTD_decompress_usingDDict(comp->dctx, dst, fd_size, src,
                                             src_size, comp->ddict_ctx);
-    else
-        retval = ZSTD_decompressDCtx(comp->dctx, *dst, dst_size, src,
-                                     src_size);
+    } else {
+        zck_log(ZCK_LOG_DEBUG, "Running decompression\n");
+        retval = ZSTD_decompressDCtx(comp->dctx, dst, fd_size, src, src_size);
+    }
+
     if(ZSTD_isError(retval)) {
-        free(*dst);
-        *dst = NULL;
         zck_log(ZCK_LOG_ERROR, "zstd decompression error: %s\n",
                 ZSTD_getErrorName(retval));
-        return False;
+        goto decomp_error_2;
     }
+    if(!zck_comp_add_to_dc(comp, dst, fd_size))
+        goto decomp_error_2;
+    free(dst);
+    free(src);
     return True;
+decomp_error_2:
+    free(dst);
+decomp_error_1:
+    free(src);
+    return False;
 }
 
 static int close(zckComp *comp) {
@@ -158,11 +174,6 @@ static int close(zckComp *comp) {
         ZSTD_freeDCtx(comp->dctx);
         comp->dctx = NULL;
     }
-    if(comp->data) {
-        free(comp->data);
-        comp->data = NULL;
-    }
-    comp->data_size = 0;
     return True;
 }
 
@@ -187,8 +198,9 @@ int zck_zstd_setup(zckComp *comp) {
     comp->init = init;
     comp->set_parameter = set_parameter;
     comp->compress = compress;
-    comp->end_chunk = end_chunk;
+    comp->end_cchunk = end_cchunk;
     comp->decompress = decompress;
+    comp->end_dchunk = end_dchunk;
     comp->close = close;
     comp->type = ZCK_COMP_ZSTD;
     return set_default_parameters(comp);
diff --git a/src/lib/dl/dl.c b/src/lib/dl/dl.c
index d9eb8e8..33edf51 100644
--- a/src/lib/dl/dl.c
+++ b/src/lib/dl/dl.c
@@ -286,7 +286,6 @@ int zck_dl_copy_src_chunks(zckRange *info, zckCtx *src, zckCtx *tgt) {
             return False;
         if(!found && !zck_range_add(info, tgt_idx, tgt))
             return False;
-
         tgt_idx = tgt_idx->next;
     }
     return True;
@@ -446,7 +445,7 @@ int zck_dl_get_header(zckCtx *zck, zckDL *dl, char *url) {
     /* Download first hundred bytes and read magic and hash type */
     if(!zck_dl_bytes(dl, url, 100, start, &buffer_len))
         return False;
-    if(!zck_read_initial(zck, dl->dst_fd))
+    if(!zck_read_initial(zck))
         return False;
     start = tell_data(dl->dst_fd);
 
@@ -456,7 +455,7 @@ int zck_dl_get_header(zckCtx *zck, zckDL *dl, char *url) {
                      start, &buffer_len))
         return False;
     /* Read and store the index hash */
-    if(!zck_read_index_hash(zck, dl->dst_fd))
+    if(!zck_read_index_hash(zck))
         return False;
     start += zck->hash_type.digest_size;
     char *digest = zck_get_index_digest(zck);
@@ -467,7 +466,7 @@ int zck_dl_get_header(zckCtx *zck, zckDL *dl, char *url) {
     zck_log(ZCK_LOG_DEBUG, "\n");
 
     /* Read and store compression type and index size */
-    if(!zck_read_ct_is(zck, dl->dst_fd))
+    if(!zck_read_ct_is(zck))
         return False;
     start = tell_data(dl->dst_fd);
     zck_log(ZCK_LOG_DEBUG, "Index size: %llu\n", zck->index_size);
@@ -476,7 +475,7 @@ int zck_dl_get_header(zckCtx *zck, zckDL *dl, char *url) {
     if(!zck_dl_bytes(dl, url, zck->index_size, start,
                      &buffer_len))
         return False;
-    if(!zck_read_index(zck, dl->dst_fd))
+    if(!zck_read_index(zck))
         return False;
 
     /* Write zeros to rest of file */
diff --git a/src/lib/header.c b/src/lib/header.c
index 671b3cf..1cd1b14 100644
--- a/src/lib/header.c
+++ b/src/lib/header.c
@@ -32,8 +32,29 @@
 
 #include "zck_private.h"
 
+#define VALIDATE(f)     if(!f) { \
+                            zck_log(ZCK_LOG_ERROR, \
+                                    "zckCtx not initialized\n"); \
+                            return False; \
+                        }
+
+#define VALIDATE_READ(f)    VALIDATE(f); \
+                            if(f->mode != ZCK_MODE_READ) { \
+                                zck_log(ZCK_LOG_ERROR, \
+                                        "zckCtx not opened for reading\n"); \
+                                return False; \
+                            }
+
+#define VALIDATE_WRITE(f)   VALIDATE(f); \
+                            if(f->mode != ZCK_MODE_WRITE) { \
+                                zck_log(ZCK_LOG_ERROR, \
+                                        "zckCtx not opened for writing\n"); \
+                                return False; \
+                            }
+
+int zck_read_initial(zckCtx *zck) {
+    VALIDATE_READ(zck);
 
-int zck_read_initial(zckCtx *zck, int src_fd) {
     char *header = zmalloc(5 + MAX_COMP_SIZE);
     size_t length = 0;
     if(header == NULL) {
@@ -43,7 +64,7 @@ int zck_read_initial(zckCtx *zck, int src_fd) {
     }
 
     zck_log(ZCK_LOG_DEBUG, "Reading magic and hash type\n");
-    if(!read_data(src_fd, header, 5 + MAX_COMP_SIZE)) {
+    if(!read_data(zck->fd, header, 5 + MAX_COMP_SIZE)) {
         free(header);
         return False;
     }
@@ -61,14 +82,18 @@ int zck_read_initial(zckCtx *zck, int src_fd) {
         return False;
     if(!zck_hash_setup(&(zck->hash_type), hash_type))
         return False;
-    if(!seek_data(src_fd, length, SEEK_SET))
+    if(!zck_hash_init(&(zck->check_full_hash), &(zck->hash_type)))
+        return False;
+    if(!seek_data(zck->fd, length, SEEK_SET))
         return False;
     zck->header_string = header;
     zck->header_size = length;
     return True;
 }
 
-int zck_read_index_hash(zckCtx *zck, int src_fd) {
+int zck_read_index_hash(zckCtx *zck) {
+    VALIDATE_READ(zck);
+
     if(zck->header_string == NULL) {
         zck_log(ZCK_LOG_ERROR,
                 "Reading index hash before initial bytes are read\n");
@@ -91,7 +116,7 @@ int zck_read_index_hash(zckCtx *zck, int src_fd) {
         return False;
     }
     zck_log(ZCK_LOG_DEBUG, "Reading index hash\n");
-    if(!read_data(src_fd, digest, zck->hash_type.digest_size)) {
+    if(!read_data(zck->fd, digest, zck->hash_type.digest_size)) {
         free(digest);
         free(header);
         return False;
@@ -106,7 +131,9 @@ int zck_read_index_hash(zckCtx *zck, int src_fd) {
     return True;
 }
 
-int zck_read_ct_is(zckCtx *zck, int src_fd) {
+int zck_read_ct_is(zckCtx *zck) {
+    VALIDATE_READ(zck);
+
     if(zck->header_string == NULL) {
         zck_log(ZCK_LOG_ERROR,
                 "Reading compression type before hash type is read\n");
@@ -124,7 +151,7 @@ int zck_read_ct_is(zckCtx *zck, int src_fd) {
         return False;
     }
     zck_log(ZCK_LOG_DEBUG, "Reading compression type and index size\n");
-    if(!read_data(src_fd, header + length, MAX_COMP_SIZE*2))
+    if(!read_data(zck->fd, header + length, MAX_COMP_SIZE*2))
         return False;
 
     int tmp = 0;
@@ -142,14 +169,16 @@ int zck_read_ct_is(zckCtx *zck, int src_fd) {
         return False;
     zck->index_size = tmp;
 
-    if(!seek_data(src_fd, length, SEEK_SET))
+    if(!seek_data(zck->fd, length, SEEK_SET))
         return False;
     zck->header_string = header;
     zck->header_size = length;
     return True;
 }
 
-int zck_read_index(zckCtx *zck, int src_fd) {
+int zck_read_index(zckCtx *zck) {
+    VALIDATE_READ(zck);
+
     char *index = zmalloc(zck->index_size);
     if(!index) {
         zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n",
@@ -157,7 +186,7 @@ int zck_read_index(zckCtx *zck, int src_fd) {
         return False;
     }
     zck_log(ZCK_LOG_DEBUG, "Reading index\n");
-    if(!read_data(src_fd, index, zck->index_size)) {
+    if(!read_data(zck->fd, index, zck->index_size)) {
         free(index);
         return False;
     }
@@ -169,19 +198,18 @@ int zck_read_index(zckCtx *zck, int src_fd) {
     return True;
 }
 
-int zck_read_header(zckCtx *zck, int src_fd) {
-    if(zck == NULL) {
-        zck_log(ZCK_LOG_ERROR, "zckCtx not initialized\n");
+int zck_read_header(zckCtx *zck) {
+    VALIDATE_READ(zck);
+
+    if(!zck_read_initial(zck))
         return False;
-    }
-    zck->fd = src_fd;
-    if(!zck_read_initial(zck, src_fd))
+    if(!zck_read_index_hash(zck))
         return False;
-    if(!zck_read_index_hash(zck, src_fd))
+    if(!zck_read_ct_is(zck))
         return False;
-    if(!zck_read_ct_is(zck, src_fd))
+    if(!zck_read_index(zck))
         return False;
-    if(!zck_read_index(zck, src_fd))
+    if(!zck_import_dict(zck))
         return False;
     return True;
 }
@@ -230,6 +258,8 @@ int zck_header_create(zckCtx *zck) {
 }
 
 int zck_write_header(zckCtx *zck) {
+    VALIDATE_WRITE(zck);
+
     if(!write_data(zck->fd, zck->header_string, zck->header_size))
         return False;
     return True;
diff --git a/src/lib/io.c b/src/lib/io.c
index ae99b4e..bde56be 100644
--- a/src/lib/io.c
+++ b/src/lib/io.c
@@ -33,22 +33,19 @@
 
 #include "zck_private.h"
 
-int read_data(int fd, char *data, size_t length) {
+ssize_t read_data(int fd, char *data, size_t length) {
     if(length == 0)
-        return True;
+        return 0;
     if(data == NULL) {
         zck_log(ZCK_LOG_ERROR, "Unable to read to NULL data pointer\n");
-        return False;
+        return -1;
     }
     ssize_t read_bytes = read(fd, data, length);
     if(read_bytes == -1) {
         zck_log(ZCK_LOG_ERROR, "Error reading data: %s\n", strerror(errno));
-        return False;
-    } else if(read_bytes != length) {
-        zck_log(ZCK_LOG_ERROR, "Short read\n");
-        return False;
+        return -1;
     }
-    return True;
+    return read_bytes;
 }
 
 int write_data(int fd, const char *data, size_t length) {
@@ -110,8 +107,9 @@ int seek_data(int fd, off_t offset, int whence) {
     return True;
 }
 
-size_t tell_data(int fd) {
-    return lseek(fd, 0, SEEK_CUR);
+ssize_t tell_data(int fd) {
+    ssize_t loc = lseek(fd, 0, SEEK_CUR);
+    return loc;
 }
 
 int chunks_from_temp(zckCtx *zck) {
diff --git a/src/lib/zck.c b/src/lib/zck.c
index 6b47440..50b45c8 100644
--- a/src/lib/zck.c
+++ b/src/lib/zck.c
@@ -39,26 +39,28 @@
                             return False; \
                         }
 
-int zck_write_file(zckCtx *zck) {
+int zck_close(zckCtx *zck) {
     VALIDATE(zck);
-    zck_index_finalize(zck);
-    zck_log(ZCK_LOG_DEBUG, "Writing header\n");
-    if(!zck_write_header(zck))
-        return False;
-    zck_log(ZCK_LOG_DEBUG, "Writing index\n");
-    if(!zck_write_index(zck))
-        return False;
-    zck_log(ZCK_LOG_DEBUG, "Writing chunks\n");
-    if(!chunks_from_temp(zck))
-        return False;
-    zck_log(ZCK_LOG_DEBUG, "Finished writing file, cleaning up\n");
-    zck_index_free(zck);
-    zck_comp_close(zck);
-    if(zck->temp_fd) {
-        close(zck->temp_fd);
-        zck->temp_fd = 0;
-    }
 
+    if(zck->mode == ZCK_MODE_WRITE) {
+        zck_index_finalize(zck);
+        zck_log(ZCK_LOG_DEBUG, "Writing header\n");
+        if(!zck_write_header(zck))
+            return False;
+        zck_log(ZCK_LOG_DEBUG, "Writing index\n");
+        if(!zck_write_index(zck))
+            return False;
+        zck_log(ZCK_LOG_DEBUG, "Writing chunks\n");
+        if(!chunks_from_temp(zck))
+            return False;
+        zck_log(ZCK_LOG_DEBUG, "Finished writing file, cleaning up\n");
+        zck_index_free(zck);
+        zck_comp_close(zck);
+        if(zck->temp_fd) {
+            close(zck->temp_fd);
+            zck->temp_fd = 0;
+        }
+    }
     return True;
 }
 
@@ -78,6 +80,7 @@ void zck_clear(zckCtx *zck) {
     zck_comp_close(zck);
     zck_hash_close(&(zck->full_hash));
     zck_hash_close(&(zck->check_full_hash));
+    zck_hash_close(&(zck->check_chunk_hash));
     zck_clear_work_index(zck);
     if(zck->full_hash_digest) {
         free(zck->full_hash_digest);
@@ -111,6 +114,57 @@ zckCtx *zck_create() {
     return zck;
 }
 
+zckCtx *zck_init_adv_read (int src_fd) {
+    zckCtx *zck = zck_create();
+    if(zck == NULL)
+        return NULL;
+
+    zck->mode = ZCK_MODE_READ;
+    zck->fd = src_fd;
+    return zck;
+}
+
+zckCtx *zck_init_read (int src_fd) {
+    zckCtx *zck = zck_init_adv_read(src_fd);
+    if(zck == NULL)
+        return NULL;
+
+    if(!zck_read_header(zck))
+        return False;
+
+    return zck;
+}
+
+zckCtx *zck_init_write (int dst_fd) {
+    zckCtx *zck = zck_create();
+    if(zck == NULL)
+        return NULL;
+
+    zck->mode = ZCK_MODE_WRITE;
+    zck->temp_fd = zck_get_tmp_fd();
+    if(zck->temp_fd < 0)
+        goto iw_error;
+
+    /* Set defaults */
+#ifdef ZCHUNK_ZSTD
+    if(!zck_set_compression_type(zck, ZCK_COMP_ZSTD))
+        goto iw_error;
+#else
+    if(!zck_set_compression_type(zck, ZCK_COMP_NONE))
+        goto iw_error;
+#endif
+    if(!zck_set_full_hash_type(zck, ZCK_HASH_SHA256))
+        goto iw_error;
+    if(!zck_set_chunk_hash_type(zck, ZCK_HASH_SHA1))
+        goto iw_error;
+    zck->fd = dst_fd;
+
+    return zck;
+iw_error:
+    free(zck);
+    return NULL;
+}
+
 int zck_set_full_hash_type(zckCtx *zck, int hash_type) {
     VALIDATE(zck);
     zck_log(ZCK_LOG_INFO, "Setting full hash to %s\n",
@@ -197,6 +251,13 @@ ssize_t zck_get_header_length(zckCtx *zck) {
     return zck->header_size + zck->index_size;
 }
 
+ssize_t zck_get_data_length(zckCtx *zck) {
+    zckIndexItem *idx = zck->index.first;
+    while(idx->next != NULL)
+        idx = idx->next;
+    return idx->start + idx->comp_length;
+}
+
 int zck_get_tmp_fd() {
     int temp_fd;
     char *fname = NULL;
@@ -231,39 +292,26 @@ int zck_get_tmp_fd() {
     return temp_fd;
 }
 
-int zck_init_write (zckCtx *zck, int dst_fd) {
+int zck_import_dict(zckCtx *zck) {
     VALIDATE(zck);
-    zck_clear(zck);
-    memset(zck, 0, sizeof(zckCtx));
 
-    zck->temp_fd = zck_get_tmp_fd();
-    if(zck->temp_fd < 0)
-        return False;
-
-    /* Set defaults */
-#ifdef ZCHUNK_ZSTD
-    zck_set_compression_type(zck, ZCK_COMP_ZSTD);
-#else
-    zck_set_compression_type(zck, ZCK_COMP_NONE);
-#endif
-    if(!zck_set_full_hash_type(zck, ZCK_HASH_SHA256))
-        return False;
-    if(!zck_set_chunk_hash_type(zck, ZCK_HASH_SHA1))
-        return False;
-    zck->fd = dst_fd;
+    size_t size = zck->index.first->length;
 
-    return True;
-}
+    /* No dict */
+    if(size == 0)
+        return True;
 
-int zck_import_dict(zckCtx *zck, char *data, size_t size) {
-    VALIDATE(zck);
-    if(data == NULL || size == 0) {
-        zck_log(ZCK_LOG_ERROR,
-                "Attempting to initialize an empty compression dictionary\n");
+    char *data = zmalloc(size);
+    if(data == NULL) {
+        zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", size);
         return False;
     }
-    zck_log(ZCK_LOG_DEBUG, "Closing compression\n");
-    if(!zck_comp_close(zck))
+    if(comp_read(zck, data, size, 0) != size) {
+        zck_log(ZCK_LOG_ERROR, "Error reading compressed dict\n");
+        return False;
+    }
+    zck_log(ZCK_LOG_DEBUG, "Resetting compression\n");
+    if(!zck_comp_reset(zck))
         return False;
     zck_log(ZCK_LOG_DEBUG, "setting dict 1\n");
     if(!zck_set_comp_parameter(zck, ZCK_COMMON_DICT, data))
@@ -271,9 +319,10 @@ int zck_import_dict(zckCtx *zck, char *data, size_t size) {
     zck_log(ZCK_LOG_DEBUG, "setting dict 2\n");
     if(!zck_set_comp_parameter(zck, ZCK_COMMON_DICT_SIZE, &size))
         return False;
-    zck_log(ZCK_LOG_DEBUG, "Initializing\n");
     if(!zck_comp_init(zck))
         return False;
+    free(data);
+
     return True;
 }
 
@@ -321,93 +370,58 @@ int zck_validate_chunk(zckCtx *zck, char *data, size_t size, zckIndexItem *idx,
     return True;
 }
 
-int zck_validate_file(zckCtx *zck) {
+int zck_validate_current_chunk(zckCtx *zck) {
     VALIDATE(zck);
-    char *digest = zck_hash_finalize(&(zck->check_full_hash));
+    char *digest = zck_hash_finalize(&(zck->check_chunk_hash));
     if(digest == NULL) {
         zck_log(ZCK_LOG_ERROR,
-                "Unable to calculate %s checksum for full file\n");
+                "Unable to calculate %s checksum for chunk\n");
         return False;
     }
-    zck_log(ZCK_LOG_DEBUG, "Checking data checksum\n");
-    if(memcmp(digest, zck->full_hash_digest, zck->hash_type.digest_size) != 0) {
+    zck_log(ZCK_LOG_DEBUG, "Checking chunk checksum\n");
+    zck_log(ZCK_LOG_DEBUG, "Expected chunk checksum: ");
+    for(int i=0; i<zck->chunk_hash_type.digest_size; i++)
+        zck_log(ZCK_LOG_DEBUG, "%02x",
+                (unsigned char)zck->comp.data_idx->digest[i]);
+    zck_log(ZCK_LOG_DEBUG, "\n");
+    zck_log(ZCK_LOG_DEBUG, "Calculated chunk checksum: ");
+    for(int i=0; i<zck->chunk_hash_type.digest_size; i++)
+        zck_log(ZCK_LOG_DEBUG, "%02x", (unsigned char)digest[i]);
+    zck_log(ZCK_LOG_DEBUG, "\n");
+    if(memcmp(digest, zck->comp.data_idx->digest,
+              zck->chunk_hash_type.digest_size) != 0) {
         free(digest);
-        zck_log(ZCK_LOG_ERROR, "Data checksum failed!\n");
+        zck_log(ZCK_LOG_ERROR, "Chunk checksum failed!\n");
         return False;
     }
-    zck_log(ZCK_LOG_DEBUG, "Data checksum valid\n");
+    zck_log(ZCK_LOG_DEBUG, "Chunk checksum valid\n");
     free(digest);
     return True;
 }
 
-int zck_decompress_to_file(zckCtx *zck, int src_fd, int dst_fd) {
+int zck_validate_file(zckCtx *zck) {
     VALIDATE(zck);
-    if(!zck_read_header(zck, src_fd))
-        return False;
-    if(!zck_hash_init(&(zck->check_full_hash), &(zck->hash_type)))
-        return False;
-
-    int start = lseek(src_fd, 0, SEEK_CUR);
-    if(start == -1) {
-        zck_log(ZCK_LOG_ERROR, "Unable to set starting point source file: %s\n",
-                strerror(errno));
+    char *digest = zck_hash_finalize(&(zck->check_full_hash));
+    if(digest == NULL) {
+        zck_log(ZCK_LOG_ERROR,
+                "Unable to calculate %s checksum for full file\n");
         return False;
     }
-
-    zckIndex *index = zck_get_index(zck);
-    zckIndexItem *idx = index->first;
-
-    /* Check if zck file is empty */
-    for(int count=0; idx; count++) {
-        size_t csize = idx->comp_length;
-        size_t size = idx->length;
-        char *cdata;
-
-        if(csize == 0) {
-            idx = idx->next;
-            continue;
-        }
-
-        cdata = zmalloc(csize);
-        if(cdata == NULL) {
-            zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", csize);
-            return False;
-        }
-        if(!seek_data(src_fd, start + idx->start, SEEK_SET))
-            return False;
-        if(!read_data(src_fd, cdata, csize)) {
-            free(cdata);
-            zck_log(ZCK_LOG_ERROR, "Error reading chunk %i\n", count);
-            return False;
-        }
-        if(!zck_validate_chunk(zck, cdata, csize, idx, count)) {
-            free(cdata);
-            return False;
-        }
-
-        char *data = NULL;
-        if(!zck_read(zck, cdata, csize, &data, size)) {
-            free(cdata);
-            zck_log(ZCK_LOG_ERROR, "Unable to decompress chunk %i\n", count);
-            return False;
-        }
-        free(cdata);
-        if(count == 0) {
-            if(!zck_import_dict(zck, data, size)) {
-                free(data);
-                return False;
-            }
-        } else {
-            if(!write_data(dst_fd, data, size)) {
-                free(data);
-                zck_log(ZCK_LOG_ERROR, "Unable to write chunk %i\n", count);
-                return False;
-            }
-        }
-        free(data);
-        idx = idx->next;
-    }
-    if(!zck_validate_file(zck))
+    zck_log(ZCK_LOG_DEBUG, "Checking data checksum\n");
+    zck_log(ZCK_LOG_INFO, "Excpected data checksum: ");
+    for(int i=0; i<zck->hash_type.digest_size; i++)
+        zck_log(ZCK_LOG_INFO, "%02x", (unsigned char)zck->full_hash_digest[i]);
+    zck_log(ZCK_LOG_INFO, "\n");
+    zck_log(ZCK_LOG_INFO, "Calculated data checksum: ");
+    for(int i=0; i<zck->hash_type.digest_size; i++)
+        zck_log(ZCK_LOG_INFO, "%02x", (unsigned char)digest[i]);
+    zck_log(ZCK_LOG_INFO, "\n");
+    if(memcmp(digest, zck->full_hash_digest, zck->hash_type.digest_size) != 0) {
+        free(digest);
+        zck_log(ZCK_LOG_ERROR, "Data checksum failed!\n");
         return False;
+    }
+    zck_log(ZCK_LOG_DEBUG, "Data checksum valid\n");
+    free(digest);
     return True;
 }
diff --git a/src/lib/zck_private.h b/src/lib/zck_private.h
index f46f1ec..de365ce 100644
--- a/src/lib/zck_private.h
+++ b/src/lib/zck_private.h
@@ -9,20 +9,23 @@
 /* Maximum string length for a compressed size_t */
 #define MAX_COMP_SIZE (((sizeof(size_t) * 8) / 7) + 1)
 
+#define ZCK_MODE_READ 0
+#define ZCK_MODE_WRITE 1
+
 #define zmalloc(x) calloc(1, x)
 
 struct zckComp;
 
 typedef int (*finit)(struct zckComp *comp);
 typedef int (*fparam)(struct zckComp *comp, int option, void *value);
-typedef int (*fcompend)(struct zckComp *comp, char **dst, size_t *dst_size,
-                        int use_dict);
-typedef int (*fcomp)(struct zckComp *comp, const char *src,
-                     const size_t src_size, char **dst, size_t *dst_size,
-                     int use_dict);
-typedef int (*fdecomp)(struct zckComp *comp, const char *src,
-                       const size_t src_size, char **dst, size_t dst_size,
-                       int use_dict);
+typedef int (*fccompend)(struct zckComp *comp, char **dst, size_t *dst_size,
+                         int use_dict);
+typedef ssize_t (*fcomp)(struct zckComp *comp, const char *src,
+                         const size_t src_size, char **dst, size_t *dst_size,
+                         int use_dict);
+typedef int (*fdecomp)(struct zckComp *comp, const int use_dict);
+typedef int (*fdcompend)(struct zckComp *comp, const int use_dict,
+                         const size_t fd_size);
 typedef int (*fcclose)(struct zckComp *comp);
 
 typedef enum log_type log_type;
@@ -76,18 +79,26 @@ typedef struct zckComp {
 
     char *data;
     size_t data_size;
+    size_t data_loc;
+    zckIndexItem *data_idx;
+    int data_eof;
+    char *dc_data;
+    size_t dc_data_size;
+    size_t dc_data_loc;
 
     finit init;
     fparam set_parameter;
     fcomp compress;
-    fcompend end_chunk;
+    fccompend end_cchunk;
     fdecomp decompress;
+    fdcompend end_dchunk;
     fcclose close;
 } zckComp;
 
 typedef struct zckCtx {
     int temp_fd;
     int fd;
+    int mode;
 
     char *full_hash_digest;
     char *header_string;
@@ -97,17 +108,24 @@ typedef struct zckCtx {
     zckIndex index;
     zckIndexItem *work_index_item;
     zckHash work_index_hash;
+
     char *index_digest;
     zckHash full_hash;
     zckHash check_full_hash;
+    zckHash check_chunk_hash;
     zckComp comp;
     zckHashType hash_type;
     zckHashType chunk_hash_type;
+
+    char *data;
+    size_t data_size;
 } zckCtx;
 
 const char *zck_hash_name_from_type(int hash_type);
 int zck_get_tmp_fd();
+int zck_import_dict(zckCtx *zck);
 int zck_validate_file(zckCtx *zck);
+int zck_validate_current_chunk(zckCtx *zck);
 void zck_clear_work_index(zckCtx *zck);
 
 /* hash/hash.h */
@@ -130,25 +148,32 @@ void zck_index_clean(zckIndex *index);
 void zck_index_free(zckCtx *zck);
 void zck_index_free_item(zckIndexItem **item);
 int zck_write_index(zckCtx *zck);
+zckIndexItem *zck_get_index_of_loc(zckIndex *index, size_t loc);
 
 /* io.c */
 int seek_data(int fd, off_t offset, int whence);
-size_t tell_data(int fd);
-int read_data(int fd, char *data, size_t length);
+ssize_t tell_data(int fd);
+ssize_t read_data(int fd, char *data, size_t length);
 int write_data(int fd, const char *data, size_t length);
 int write_comp_size(int fd, size_t val);
 int read_comp_size(int fd, size_t *val, size_t *length);
 int chunks_from_temp(zckCtx *zck);
 
 /* header.c */
-int zck_read_initial(zckCtx *zck, int src_fd);
-int zck_read_index_hash(zckCtx *zck, int src_fd);
-int zck_read_ct_is(zckCtx *zck, int src_fd);
-int zck_read_index(zckCtx *zck, int src_fd);
+int zck_read_initial(zckCtx *zck);
+int zck_read_index_hash(zckCtx *zck);
+int zck_read_ct_is(zckCtx *zck);
+int zck_read_index(zckCtx *zck);
+int zck_read_header(zckCtx *zck);
 int zck_header_create(zckCtx *zck);
-int zck_read_header(zckCtx *zck, int src_fd);
 int zck_write_header(zckCtx *zck);
 
+/* comp/comp.c */
+int zck_comp_add_to_dc(zckComp *comp, const char *src, size_t src_size);
+int zck_comp_add_to_data(zckComp *comp, const char *src, size_t src_size);
+size_t zck_comp_read_from_dc(zckComp *comp, char *dst, size_t dst_size);
+ssize_t comp_read(zckCtx *zck, char *dst, size_t dst_size, int use_dict);
+
 /* dl/range.c */
 char *zck_range_get_char(zckRangeItem **range, int max_ranges);
 int zck_range_add(zckRange *info, zckIndexItem *idx, zckCtx *zck);
@@ -168,7 +193,6 @@ int zck_compint_to_int(int *val, const char *compint, size_t *length);
 int zck_compint_to_size(size_t *val, const char *compint, size_t *length);
 
 
-
 /* log.c */
 void zck_log(log_type lt, const char *format, ...);
 #endif